{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradiant Boosting Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.datasets import load_breast_cancer\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "357 out of 569 persons have cancer\n",
      "features 30\n"
     ]
    }
   ],
   "source": [
    "cancer=load_breast_cancer()\n",
    "n_samples = len(cancer.target)\n",
    "X = cancer.data\n",
    "y = cancer.target\n",
    "total=len(y)\n",
    "has_cancer=np.sum(y)\n",
    "no_cancer=total-has_cancer\n",
    "print(f'{has_cancer} out of {total} persons have cancer')\n",
    "print(f'features {X.shape[1]}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 1 Log of Odds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the log_of_odds is : 0.5211495071076265\n",
      "the probability is : 0.6274165202108963\n"
     ]
    }
   ],
   "source": [
    "log_of_odds0=np.log(has_cancer / no_cancer)\n",
    "probability0=np.exp(log_of_odds0)/(np.exp(log_of_odds0)+1)\n",
    "print(f'the log_of_odds is : {log_of_odds0}')\n",
    "print(f'the probability is : {probability0}')\n",
    "predict0=1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Step 2 For Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss is 375.7200026920845\n",
      "loss is 375.72000269208445\n",
      "gamma1 1.221364201577474\n",
      "gamma2 -2.4363001705150813\n",
      "new loss is 338.13703383104206\n"
     ]
    }
   ],
   "source": [
    "#explore\n",
    "residuals0=y - probability0\n",
    "loss0=-(y*np.log(probability0)+(1-y)*np.log(1-probability0))\n",
    "print(f'loss is {loss0.sum()}')\n",
    "loss0=-y * log_of_odds0 + np.log(1+np.exp(log_of_odds0))\n",
    "print(f'loss is {loss0.sum()}')\n",
    "\n",
    "tree0 = DecisionTreeRegressor(max_depth=1)\n",
    "tree0.fit(X, residuals0)\n",
    "stem = tree0.tree_\n",
    "\n",
    "leaf_index=tree0.apply(X)\n",
    "#for leaf 1\n",
    "n_leaf1=len(leaf_index[leaf_index==1])\n",
    "previous_probability1 = np.array([probability0] * n_leaf1)\n",
    "denominator1 = np.sum(previous_probability1 * (1-previous_probability1))\n",
    "gamma1 = stem.value[1][0][0] * n_leaf1 / denominator1\n",
    "print(f'gamma1 {gamma1}')\n",
    "#for leaf 2\n",
    "n_leaf2=len(leaf_index[leaf_index==2])\n",
    "previous_probability2 = np.array([probability0] * n_leaf2)\n",
    "denominator2 = np.sum(previous_probability2 * (1-previous_probability2))\n",
    "gamma2 = stem.value[2][0][0] * n_leaf2 / denominator2\n",
    "print(f'gamma2 {gamma2}')\n",
    "\n",
    "gamma_value=[gamma1, gamma2]\n",
    "gamma = np.array([gamma_value[index-1] for index in leaf_index])\n",
    "new_log_of_odds = log_of_odds0 + 0.1 * gamma\n",
    "new_loss=-y * new_log_of_odds + np.log(1+np.exp(new_log_of_odds))\n",
    "print(f'new loss is {new_loss.sum()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#now let's get started\n",
    "n_times = 20\n",
    "learning_rate=0.1\n",
    "log_of_odds=np.zeros([n_times+1,n_samples])\n",
    "\n",
    "residuals = np.zeros([n_times+1,n_samples])\n",
    "loss = [0] * (n_times+1)\n",
    "prediction = np.zeros([n_times+1,n_samples])\n",
    "score = np.zeros([n_times+1])\n",
    "#calculation\n",
    "log_of_odds[0] = [log_of_odds0] * n_samples\n",
    "prediction[0] = [probability0] * n_samples\n",
    "score[0]=np.sum((prediction[0]>0.5)*1.0==y) / n_samples\n",
    "residuals[0] = y - prediction[0]\n",
    "loss[0] = loss0.sum()\n",
    "trees = []\n",
    "gamma_value = np.zeros([n_times+1,8])\n",
    "gamma = np.zeros([n_times+1,n_samples])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(n_times):\n",
    "    dt = DecisionTreeRegressor(max_depth=2)\n",
    "    dt=dt.fit(X, residuals[i])\n",
    "    tree = dt.tree_\n",
    "    trees.append(tree)\n",
    "    leaf_indeces=dt.apply(X)\n",
    "    unique_leafs=np.unique(leaf_indeces)\n",
    "    n_leaf=len(unique_leafs)\n",
    "    #for leaf 1\n",
    "    for ileaf in range(n_leaf):\n",
    "        leaf_index=unique_leafs[ileaf]\n",
    "        n_leaf=len(leaf_indeces[leaf_indeces==leaf_index])\n",
    "        previous_probability = prediction[i][leaf_indeces==leaf_index]\n",
    "        denominator = np.sum(previous_probability * (1-previous_probability))\n",
    "        igamma = tree.value[ileaf+1][0][0] * n_leaf / denominator\n",
    "        gamma_value[i][ileaf]=igamma\n",
    "\n",
    "    gamma[i] = [gamma_value[i][np.where(unique_leafs==index)] for index in leaf_indeces]\n",
    "    #next \n",
    "    log_of_odds[i+1] = log_of_odds[i] + learning_rate * gamma[i]\n",
    "\n",
    "    prediction[i+1] = np.array([np.exp(odds)/(np.exp(odds)+1) for odds in log_of_odds[i+1]])\n",
    "    score[i+1]=np.sum((prediction[i+1]>0.5)*1.0==y) / n_samples\n",
    "    residuals[i+1] = y - prediction[i+1]\n",
    "    loss[i+1]=np.sum(-y * log_of_odds[i+1] + np.log(1+np.exp(log_of_odds[i+1])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x21a2acdc9e8>]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VPW9//HXJzsJkBASQoCwh13WiCBFUKvGpWKrVlurqG1xrdbbq3a5bW17e/tTr9p6WxdcKu5rVdxwl1BFMCBgIhDCHiALa4BAQpLv74852BQTspDkTGbez8djHnPme87JfHJm8p6T75zzPeacQ0REQleE3wWIiEjbUtCLiIQ4Bb2ISIhT0IuIhDgFvYhIiFPQi4iEOAW9iEiIU9CLiIQ4Bb2ISIiL8rsAgJSUFNe/f3+/yxAR6VCWLFmy3TmX2thyQRH0/fv3Jzc31+8yREQ6FDPb2JTl1HUjIhLiFPQiIiFOQS8iEuIU9CIiIU5BLyIS4hT0IiIhTkEvIhLiOnTQb9pRwW1z8zlUU+t3KSIiQatDB31ByV4e+2QDzyze5HcpIiJBq0MH/anDezBpYDJ/fm8N5QcP+V2OiEhQ6tBBb2b86qwR7NxfxQMfrfW7HBGRoNShgx7guD6JfHtcbx7553q27j7gdzkiIkGnwwc9wM9OH4ID/ved1X6XIiISdEIi6Pt0i+fKKQN4+fMt5G3Z43c5IiJBJSSCHuDakweR1CmaP76xEuec3+WIiASNkAn6rnHR3HhqJgvX7eDD1aV+lyMiEjRCJugBvn9CPwakJPCnN1dRrZOoRESAEAv6mKgIbs0exprSfTyfW+R3OSIiQSGkgh7gjJFpZPXrxt3vFrCvstrvckREfBdyQW9m/Ors4WzfV8nsnHV+lyMi4ruQC3qAcX27cc7odGbnrKV4z0G/yxER8VVIBj3ArdnDqK2Fu9/VSVQiEt5CNugzkuO5bHI/XlhSxMpt5X6XIyLim5ANeoDrTxlM17ho/vTWKr9LERHxTaNBb2ZxZrbYzJabWb6Z/c5rf8zM1pvZMu821ms3M7vXzArNbIWZjW/rX6IhSfEx/OSUweQUlDG/oMyvMkREfNWUPfpK4BTn3BhgLJBtZpO8eTc758Z6t2Ve25lApnebBdzf2kU3x6WT+5GR3Ik/vbmSmloNjSAi4afRoHcB+7yH0d7taIk5A3jcW+9TIMnM0o+91JaJjYrk1uxhrCrey0tLdRKViISfJvXRm1mkmS0DSoF3nXOLvFl/9Lpn7jGzWK+tN7C5zupFXtuRP3OWmeWaWW5ZWdt2q5x9XDpjM5K4653VVFTpJCoRCS9NCnrnXI1zbizQB5hoZqOAXwDDgOOBZOBWb3Gr70fU8zNnO+eynHNZqampLSq+qcyM/zp7OCXllTy8YH2bPpeISLBp1lE3zrndwEdAtnNum9c9Uwn8HZjoLVYEZNRZrQ+wtRVqPSZZ/ZPJHtmTB+avpXSvTqISkfDRlKNuUs0syZvuBHwTWHW4393MDDgPyPNWmQtc5h19MwnY45zb1ibVN9OtZw6jqrqWP7+3xu9SRETaTVP26NOBD81sBfAZgT7614GnzOwL4AsgBfhvb/k3gXVAIfAQcG2rV91CA1IS+MGkfjy7eBNrSvb6XY6ISLuwYLgaU1ZWlsvNzW2X59q5v4ppd3zI8QOSefTy49vlOUVE2oKZLXHOZTW2XEifGVuf5IQYrjtlMB+sKuWTwu1+lyMi0ubCLugBLj+xP72TOvHHN1dSq5OoRCTEhWXQx0VHcvMZQ8nfWs4LSzY3voKISAcWlkEPcO6YXkzsn8wf31hJSbkOtxSR0BW2QR8RYdx+wWgqq2v51ct5BMOX0iIibSFsgx4Ch1v+7PQhvLeyhNdWBMWh/iIirS6sgx7gh98YyJiMJG6bm8+OfZV+lyMi0urCPugjI4w7LxjN3oOHuO21L/0uR0Sk1YV90AMMSevCT07J5LXlW3knv9jvckREWpWC3nPN9EEM69mF/3oljz0HDvldjohIq1HQe6IjI7jzgjHs2F/FH99QF46IhA4FfR3H9UnkqpMG8nxuETm6xqyIhAgF/RFuODWTQakJ/OIfX7CvUlejEpGOT0F/hLjoSO64YDRb9xzgjnmr/C5HROSYKejrMaFfMpef2J/HF25k0bodfpcjInJMFPQNuPmMoWQkd+LWl1ZwoKrG73JERFpMQd+A+Jgobv/OaDbsqOCe9wr8LkdEpMUU9Edx4uAUvjexLw8vWMeyzbv9LkdEpEUU9I34xVnDSOsaxy0vLqeyWl04ItLxKOgb0TUumv/59nEUlOzjbx+u9bscEZFmU9A3wcnDevCdcb2578NCvtxa7nc5IiLNoqBvot98awRJ8THc8tJyqmtq/S5HRKTJFPRNlBQfwx9mjCRvSzmzF6zzuxwRkSZT0DfDmcelc9ZxPfnze2soLN3ndzkiIk2ioG+m3507iviYSG55cTk1tbrOrIgEPwV9M6V2ieW33xrB0k27efSf6/0uR0SkUQr6FjhvbG9OH5HG7fNWsWTjLr/LERE5KgV9C5gZd144hl5JnbjuqaVs10XFRSSINRr0ZhZnZovNbLmZ5ZvZ77z2AWa2yMzWmNlzZhbjtcd6jwu9+f3b9lfwR2KnaO7/wXh2VVRx47Ofq79eRIJWU/boK4FTnHNjgLFAtplNAm4H7nHOZQK7gB96y/8Q2OWcGwzc4y0Xkkb2SuQP543i48Id3POuBj4TkeDUaNC7gMPHEkZ7NwecArzotc8BzvOmZ3iP8eafambWahUHme9mZXDx8Rn89cNC3l9Z4nc5IiJf06Q+ejOLNLNlQCnwLrAW2O2cO3ytvSKgtzfdG9gM4M3fA3Sv52fOMrNcM8stK+vY12e97dyRjOrdlZueW8bmnRV+lyMi8m+aFPTOuRrn3FigDzARGF7fYt59fXvvX+vAds7Nds5lOeeyUlNTm1pvUIqLjuT+SyYAcPWTSzh4SKNcikjwaNZRN8653cBHwCQgycyivFl9gK3edBGQAeDNTwR2tkaxwSwjOZ57LhpL/tZybpub73c5IiJfacpRN6lmluRNdwK+CawEPgQu8BabCbzqTc/1HuPN/8A5FxaHpJw6PI3rTh7Es59t5vnczX6XIyICQFTji5AOzDGzSAIfDM875143sy+BZ83sv4HPgUe85R8BnjCzQgJ78he3Qd1B6z9OG8qyzbv59St5jOzVlZG9Ev0uSUTCnAXDznZWVpbLzc31u4xWs31fJefc+09ioiJ47SffILFTtN8liUgIMrMlzrmsxpbTmbFtIKVzLH+7ZDxbdx/gZ88vp1YnU4mIjxT0bWRCv2786uzhvLeyhAdzNH69iPhHQd+GLj+xP+eMTufOt1fxydrtfpcjImFKQd+GzIzbzx/NgJQEbnjmc4r3HPS7JBEJQwr6NpYQG8UDP5hARVUN1z+9lEO63qyItDMFfTvITOvC/zt/NLkbd/H/3lrldzkiEmYU9O3k3DG9uPzE/jzyz/W8sWKb3+WISBhR0LejX541nHF9k7jlxeWsLdPFxUWkfSjo21FMVAT3XTKe2OhIZj2ey+6KKr9LEpEwoKBvZ+mJnbj/kvFs3nmAH83J1UiXItLmFPQ+OGFgd+6+aAxLNu3ip88u02UIRaRNKeh9cs7oXvz67BHMyy/md6/lEwxjDolIaGrK6JXSRq78xgC27TnAQwvWk57YiWumD/K7JBEJQQp6n/3izOEUl1dy+7xV9EyM5dvj+vhdkoiEGAW9zyIijP+9cDTb91Zy8wsrSOkcy9TMjn1pRREJLuqjDwKxUZE8eNkEBvfozNVPLCFvyx6/SxKREKKgDxJd46J57IqJJHaK5orHPmPzzgq/SxKREKGgDyI9E+N47MqJVB6qYebfF7Nrv06oEpFjp6APMkPSuvDwzOMp2nWAHz2uE6pE5Ngp6IPQxAHJ/PmisSzdtIsbnvlcJ1SJyDFR0Aeps45L5zfnjOCdL0u4ba5OqBKRltPhlUHsiikDKN5zkAdz1pGeFMe10wf7XZKIdEAK+iB3a/YwissPcse81aR1ieP8CTqhSkSaR0Ef5CIijDsvGEPZ3kpufWkFqV1iOWmITqgSkaZTH30HEBMVwQOXBk6ouuZJnVAlIs2joO8gusZFM+fKiSTFx3D53z9jdfFev0sSkQ5CQd+BpHWNY86VE4mKML774EKWbtrld0ki0gEo6DuYwT0688LVk+kWH80lDy1iwZoyv0sSkSDXaNCbWYaZfWhmK80s38xu9NpvM7MtZrbMu51VZ51fmFmhma02szPa8hcIRxnJ8Tx/9WT6dY/nysc+460vtvldkogEsabs0VcDP3PODQcmAdeZ2Qhv3j3OubHe7U0Ab97FwEggG7jPzCLboPaw1qNLHM9dNZnRfZK47umlPP/ZZr9LEpEg1WjQO+e2OeeWetN7gZVA76OsMgN41jlX6ZxbDxQCE1ujWPl3iZ2ieeKHE5mamcotL63goZx1fpckIkGoWX30ZtYfGAcs8pquN7MVZvaomXXz2noDdXcvi6jng8HMZplZrpnllpWpn7ml4mOieOiyLM4enc4f31zJnW+v0nAJIvJvmhz0ZtYZeAn4qXOuHLgfGASMBbYBdx1etJ7Vv5Y8zrnZzrks51xWaqpOADoWMVER3HvxOL43sS9/+3Atv341j1oNhCYiniadGWtm0QRC/inn3D8AnHMldeY/BLzuPSwCMuqs3gfY2irVSoMiI4z/+fYoEjtF88D8tZQfqOau744hOlIHVomEu6YcdWPAI8BK59zdddrT6yz2bSDPm54LXGxmsWY2AMgEFrdeydIQM+PnZw7j1uxhzF2+lVmP53KgSuPZi4S7puzRTwEuBb4ws2Ve2y+B75nZWALdMhuAqwCcc/lm9jzwJYEjdq5zzilt2tE10weRFB/NL1/+gsseXcTDM48nsVO032WJiE8sGL64y8rKcrm5uX6XEXJeX7GVm55bRmaPLsy5ciKpXWL9LklEWpGZLXHOZTW2nDpwQ9g5o3vx8MzjWb99P999cCFFu3TBcZFwpKAPcdOGpPLkjyayY18lFz6wkMLSfX6XJCLtTEEfBib0S+a5qyZzqMZpMDSRMKSgDxPD07vy4tWT6RIXxcWzP+W15TriVSRcKOjDSP+UBF6+dgpj+iTyk2c+59731+gsWpEwoKAPM8kJMTz5oxP4zrje3P1uAf/x/HIqq3X0q0go0zVjw1BsVCR3fXcMA1ISuOvdAop2VfDgpVkkJ8T4XZqItAHt0YcpM+Mnp2by1++PY3nRHs7728c6IkckRCnow9w5o3vx7KxJVFRV8+37Pubjwu1+lyQirUxBL4zv242Xr51CemIcMx9dzDOLN/ldkoi0IgW9AIHLE750zYlMGZzCL/7xBf/z5kpqNNSxSEhQ0MtXusRF88jMLC6b3I/ZOeu4+sklVFRV+12WiBwjBb38m6jICH4/YxS3fWsE768s4cIHFlK856DfZYnIMVDQS70unzKAR2Yez4bt+5nxt3+St2WP3yWJSAsp6KVBJw/rwYvXnEikGRc+sJB38ov9LklEWkBBL0c1PL0rr1w/hSFpnbnqySXc+/4afUkr0sEo6KVRPbrE8eysycwY04u73y3gkoc/Vb+9SAeioJcm6RQTyT0XjeXOC0azfPMezvxLDh+sKml8RRHxnYJemszMuDArg9d+8g16Jnbiysdy+f1rX2pQNJEgp6CXZhvcozMvX3sil5/Yn0c/Xs/593/C+u37/S5LRBqgoJcWiYuO5LZzRzL70gkU7TrAOfcu4OXPi/wuS0TqoaCXY3L6yJ68ecNURvZK5KbnlvMfzy9jf6XOphUJJgp6OWa9kjrx9I9P4MZTM3nl8y2c8386wUokmCjopVVERUZw02lDePrHkzhQVcN37vuER/+5XpcqFAkCCnppVZMGduetG6dy0pAUfv/6l/z48Vx27q/yuyyRsKagl1bXLSGGhy7L4rffGkFOwXbO/EsOC9fu8LsskbCloJc2YWZcMWUA/7j2RBJiovj+w59yx7xVHDykY+5F2puCXtrUqN6JvPaTb3DhhD7c99Fazr53AUs27vS7LJGw0mjQm1mGmX1oZivNLN/MbvTak83sXTNb491389rNzO41s0IzW2Fm49v6l5DglhAbxR0XjGHOlRM5eKiWCx5YyG1z83UYpkg7acoefTXwM+fccGAScJ2ZjQB+DrzvnMsE3vceA5wJZHq3WcD9rV61dEjThqTy9k0ncdmkfsxZuIEz/pzDgjVlfpclEvIaDXrn3Dbn3FJvei+wEugNzADmeIvNAc7zpmcAj7uAT4EkM0tv9cqlQ+ocG8XvZozi+asmExMVwaWPLObmF5azp+KQ36WJhKxm9dGbWX9gHLAISHPObYPAhwHQw1usN7C5zmpFXpvIV47vn8ybN0zl2umD+MfnW/jmPfOZl7fN77JEQlKTg97MOgMvAT91zpUfbdF62r521oyZzTKzXDPLLSvTv+/hKC46kluyh/HqdVNI7RzL1U8u5Zonl1C6V2Pdi7SmJgW9mUUTCPmnnHP/8JpLDnfJePelXnsRkFFn9T7A1iN/pnNutnMuyzmXlZqa2tL6JQSM6p3Iq9dP4Zbsoby/qpTT7s7hxSVFOqtWpJU05agbAx4BVjrn7q4zay4w05ueCbxap/0y7+ibScCew108Ig2Jjozg2umDeevGqQxJ68x/vrCcmX//jKJdFX6XJtLhWWN7TWb2DWAB8AVQ6zX/kkA//fNAX2ATcKFzbqf3wfBXIBuoAK5wzuUe7TmysrJcbu5RF5EwUlvreHLRRm5/axUOuDV7GJdO6kdERH29giLhy8yWOOeyGl0uGP49VtBLfYp2VfDLl/PIKShjXN8kfn3OCMb37eZ3WSJBo6lBrzNjJWj16RbPnCuO564Lx1C06wDfue8Trn96KZt2qDtHpDmi/C5A5GjMjPMn9CF7VE9m56xjds463s4vZubk/lx/ymCS4mP8LlEk6GmPXjqEhNgobjptCB/dPJ3vjOvDIx+vZ9qdH/HwgnW6OLlIIxT00qGkdY3j9gtG8+YNUxmTkcR/v7GS0+7O4Y0V23Q4pkgDFPTSIQ1P78rjV05kzpUTiY+J5Lqnl3L+/Z9oZEyReijopUObNiSVN26Yyh3nj6Zo1wHOv38h1z61hA3b9/tdmkjQ0OGVEjIqqqp5KGc9D+as5VBNLT+Y1I8bTsmkW4K+sJXQpMMrJezEx0Rx4zcz+eg/p3PBhD7M+WQDJ935IbNz1urKVhLWtEcvIWt18V7+9NZKPlpdRlrXWK4/JZOLsjKIidL+jYQG7dFL2BvaswuPXTGRZ348iYxu8fz6lTxOuesjXsjdTHVNbeM/QCREKOgl5E0e1J0Xrp7MY1ccT7f4GG5+cQWn35PD3OVbqa31/z9akbamoJewYGZMH9qDuddP4YEfTCAq0rjhmc85694FvJNfrGPwJaQp6CWsmBnZo3ry1o0n8ZeLx1JZXcusJ5Zw3t8+JqegTIEvIUlBL2EpMsKYMbY37950EnecP5rt+6q47NHFXPTgpyxat8Pv8kRalY66EQEqq2t4/rPN/N8HhZTurWRqZgo/O30oYzOS/C5NpEEaj16kBQ5U1fDkpxu5f/5adu6v4pvD07h62kAm9OtG4Jo6IsFDQS9yDPZVVvPYx+uZnbOO8oPVDE/vymWT+zFjbC/iYzS6twQHBb1IK6ioquaVz7fy+MINrCreS9e4KC7MyuDSSf3on5Lgd3kS5hT0Iq3IOcdnG3bx+MINzMsrprrWcdKQVGZO7sf0oT2I1PVsxQdNDXr9DyrSBGbGxAHJTByQTGn5QZ5evImnF23ih3Ny6dOtEz+Y1I+LsjI0gJoEJe3Ri7TQoZpa3skvYc7CDSxev5PYqAi+NaYXl03ux+g+OlpH2p66bkTa0aricp5YuJGXP99CRVUNYzOSuGxyP84enU5sVKTf5UmIUtCL+KD84CFeWlLEEws3sm77ftIT47hm+iC+m5VBXLQCX1qXgl7ER7W1jpw1Zfz1g0JyN+6iZ9dA4F90vAJfWo+CXiQIOOf4ZO0O/vxeAZ9t2EVa11iumTaIiyf2VeDLMVPQiwQR5xwL1+7gz++tYfGGnaR1jeXqaYP4ngJfjoGCXiQIOedYuM4L/PU76dElEPjfP0GBL82noBcJcgu9Lp1F63eS6gX+JQp8aYZWu5SgmT1qZqVmllen7TYz22Jmy7zbWXXm/cLMCs1stZmd0fJfQSS0TR7UneeumswzP57EoNQE/vD6l0y940MeXrCOA1W6mLm0nkb36M3sJGAf8LhzbpTXdhuwzzn3v0csOwJ4BpgI9ALeA4Y45476rtUevQgsWreDv7y/hk/W7iClcyxXTOnPjLG96NMt3u/SJEi12hAIzrkcM+vfxOedATzrnKsE1ptZIYHQX9jE9UXC1gkDu/P0wO4sXr+Te99fw51vr+bOt1dzfP9uzBjbm7OPS9cQC9Iix3KFqevNbIXXtdPNa+sNbK6zTJHXJiJNNHFAMk/+6AQW3HIyN58xlN0Vh/ivV/I4/o/v8cPHPmPu8q1UVFX7XaZ0IC0d1Ox+4A+A8+7vAq4E6hvCr96+ITObBcwC6Nu3bwvLEAldGcnxXHfyYK6dPogvt5Uzd9lW5i7fyvurSomPieSMkT05d2wvpg5OISpSVwWVhrUo6J1zJYenzewh4HXvYRGQUWfRPsDWBn7GbGA2BProW1KHSDgwM0b2SmRkr0RuzR7G4g07eXXZFt5YsY2XP99C94QYzhmdzrljezO+b5KuhCVf06TDK70++tfrfBmb7pzb5k3fBJzgnLvYzEYCT/OvL2PfBzL1ZaxI66usrmH+6jJeXbaV91aWUFldS0ZyJ2aM6c2Msb3ITOvid4nSxlrtOHozewaYDqQAJcBvvcdjCXTLbACuqhP8vyLQjVMN/NQ591ZjRSjoRY7N3oOHeDu/hFeXbeHjwu3UOhiUmsCZo9LJHtWTkb26ak8/BOmEKZEwVbr3IPPyipmXV8yi9TupqXX06daJ7JE9yR7Vk/F9uxGhK2KFBAW9iLBzfxXvfVnCvPxi/rlmO1U1taR2ieWMkWlkj0znhIHJROuL3A5LQS8i/6b84CE+XFXKvLxiPlpdxoFDNSTFR/PN4WmcOaonUwanaPiFDkZBLyINOlBVw/yCMt7OL+a9lSXsPVhNQkwkJw/rQfaonkzNTCWxU7TfZUojdHFwEWlQp5hIskcF+uyrqmv5ZO123s4v5p38El5fsY3ICGN83ySmD+3BtCGpjEjvqn79Dkx79CLylZpax5KNu5hfUMr8gjLytpQDkNI5lpOGpDBtSConZaZqKIYgoa4bETlmpXsPsqBgO/MLyshZU8buikOYwZg+SUwbksr0oamM7pNEpPb2faGgF5FWVVPrWFG0m/kFZXy0uozlRbtxDpLio5mamcr0IalMHZJCjy5xfpcaNhT0ItKmdu2vYkHhdj5aXUpOQRnb91UBMLJX10AXz5BUJvTrpsM325CCXkTaTW2t48tt5cwvKGN+QRlLNu6iptbROTaKEwd1Z9rQQN9+RrLG1m9NCnoR8U35wUN8Urgj0LdfUMaW3QcAGJiawLQhqUwbksqkgd113P4xUtCLSFBwzrG2bP9Xe/uL1u2gsrqW2KgIJg5I/upL3UGpnTUeTzMp6EUkKB08VMOi9TuZv7qM+QWlrC3bD0CvxDgm9E8ms0dnBvfoTGaPzvTrnkBMlPr4G6ITpkQkKMVFR37VfQMjKNpVQU7BduYXlPL5pl28tvxfl7CIijD6dY8ns0eXQPindWZQauDWKUbdPk2lPXoRCSoVVdWsK9vPmtK9FJbuY03JPgpL97FxZwU1tYG8MoOMbvFf7fkP6tGZ0X0SGZrWJay6f7RHLyIdUnxMFKN6JzKqd+K/tVdW17Bhe0Ug/L0PgcLSfV+NygnQr3s82SN7csaonoztk6RhGzzaoxeRDq26ppZNOytYtH4n8/KK+WTtdg7VOHp2jeOMkWmcMaonE/snh+R1dfVlrIiEpT0HDvHBqhLm5RUzv6CMg4dqSU6I4bThaWSP6smJg7sTGxUa/fsKehEJexVV1cxfXca8/GI+WFnK3spqusRGccrwHmSP7Mm0oanEx3TcHmz10YtI2IuPieLM49I587h0Kqtr+GTtDuZ9Ucy7K0t4ddlWYqMimDYkldNGpDGubxIDUjqH5ABt2qMXkbBTXVPLZxt2MS9vG2/nl1BcfhCA+JhIRvbqysheiRzXO5Hj+iQyMCUhaPv31XUjItIEtbWOgtK9fFG0h7wte8jbWs6XW8s5cKgGgLjoCEakd+U470igUb0TyezROSjCX0EvItJCNbWOtWX7yNuyhy+2BD4A8reWU1EVCP/YqAiGp3dlVO/AB8CQtC5kpnWhc2z79oYr6EVEWlFNrWP99v2BvX7vAyB/azn7Kqu/WqZXYhyZaV3I7NGZIWldGJwWOKGrS1zbXH9XX8aKiLSiyAhjsDcOz3njegOBbp+NOytYU7KXNaX7vrr/1Bu47bD0xDgGe+Gf2aNz4MMgrTNd2+gD4EgKehGRFoqIMAakJDAgJYHTR/6rvabWUbSrgoIS7yzekn0UlO7lqUUbOXjoXx8APbvG8aOpA/jR1IFtWqeCXkSklUVGGP26J9CvewKnjUj7qr221rFl9wEKvD3/gpK9pHaJbfN6FPQiIu0kIsLISI4nIzmeU4enNb5Caz1vuz2TiIj4otGgN7NHzazUzPLqtCWb2btmtsa77+a1m5nda2aFZrbCzMa3ZfEiItK4puzRPwZkH9H2c+B951wm8L73GOBMINO7zQLub50yRUSkpRoNeudcDrDziOYZwBxveg5wXp32x13Ap0CSmaW3VrEiItJ8Le2jT3PObQPw7nt47b2BzXWWK/LaRETEJ639ZWx9w77Ve+qtmc0ys1wzyy0rK2vlMkRE5LCWBn3J4S4Z777Uay8CMuos1wfYSj2cc7Odc1nOuazU1NQWliEiIo1padDPBWZ60zOBV+u0X+YdfTMJ2HO4i0dERPzR6KBmZvYMMB1IAUqA3wKvAM8DfYFNwIXOuZ2/IzmmAAAFGElEQVQWuPz6XwkcpVMBXOGca3S0MjMrAza28HdIAba3cN22FKx1QfDWprqaR3U1TyjW1c8512iXSFCMXnkszCy3KaO3tbdgrQuCtzbV1Tyqq3nCuS6dGSsiEuIU9CIiIS4Ugn623wU0IFjrguCtTXU1j+pqnrCtq8P30YuIyNGFwh69iIgcRYcJejPLNrPV3siYP69nfqyZPefNX2Rm/duhpgwz+9DMVppZvpndWM8y081sj5kt826/aeu6vOfdYGZfeM/5tUNc/Rhp1MyG1tkOy8ys3Mx+esQy7ba9mjMyaz3rzvSWWWNmM+tbppXrutPMVnmv1ctmltTAukd93dugrtvMbEud1+usBtY96t9vG9T1XJ2aNpjZsgbWbZPt1VA2+Pb+cs4F/Q2IBNYCA4EYYDkw4ohlrgUe8KYvBp5rh7rSgfHedBegoJ66pgOv+7DNNgApR5l/FvAWgWErJgGLfHhNiwkcB+zL9gJOAsYDeXXa7gB+7k3/HLi9nvWSgXXefTdvulsb13U6EOVN315fXU153dugrtuA/2zCa33Uv9/WruuI+XcBv2nP7dVQNvj1/uooe/QTgULn3DrnXBXwLIGRMuuqO6Lmi8Cp3glcbcY5t805t9Sb3guspOMM4ub3SKOnAmudcy09Ue6YueaNzFrXGcC7zrmdzrldwLt8fSjvVq3LOfeOc67ae/gpgeFF2lUD26spmvL32yZ1eRnwXeCZ1nq+JtbUUDb48v7qKEHflFExv1rG+4PYA3Rvl+oAr6toHLContmTzWy5mb1lZiPrmd8WHPCOmS0xs1n1zPd7pNGLafiPz4/tdVhDI7PW5fe2u5LAf2P1aex1bwvXe11KjzbQFeHn9poKlDjn1jQwv8231xHZ4Mv7q6MEfVNGxWzyyJmtzcw6Ay8BP3XOlR8xeymB7okxwP8RGD6iPUxxzo0ncDGY68zspCPm+7m9YoBzgRfqme3X9moOP7fdr4Bq4KkGFmnsdW9t9wODgLHANgLdJEfybXsB3+Poe/Ntur0ayYYGV6un7Zi2V0cJ+qaMivnVMmYWBSTSsn8zm8XMogm8kE855/5x5HznXLlzbp83/SYQbWYpbV2Xc26rd18KvEzg3+e6mjzSaBs4E1jqnCs5coZf26uOhkZmrcuXbed9KXcOcInzOnOP1ITXvVU550qcczXOuVrgoQaez6/tFQV8B3iuoWXacns1kA2+vL86StB/BmSa2QBvb/BiAiNl1lV3RM0LgA8a+mNoLV7/3yPASufc3Q0s0/PwdwVmNpHANt/RxnUlmFmXw9MEvsjLO2IxP0cabXAvy4/tdYSGRmat623gdDPr5nVVnO61tRkzywZuBc51zlU0sExTXvfWrqvu9zrfbuD5mvL32xa+CaxyzhXVN7Mtt9dRssGf91drf9vcVjcCR4kUEPj2/lde2+8JvPEB4gh0BRQCi4GB7VDTNwj8S7UCWObdzgKuBq72lrkeyCdwpMGnwIntUNdA7/mWe899eHvVrcuAv3nb8wsgq51ex3gCwZ1Yp82X7UXgw2YbcIjAXtQPCXyv8z6wxrtP9pbNAh6us+6V3nutkMAorW1dVyGBftvD77PDR5j1At482uvexnU94b1/VhAIsfQj6/Ief+3vty3r8tofO/y+qrNsu2yvo2SDL+8vnRkrIhLiOkrXjYiItJCCXkQkxCnoRURCnIJeRCTEKehFREKcgl5EJMQp6EVEQpyCXkQkxP1//YyCO8CnwysAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x21a2ad85240>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xt0HGeZ5/Hvo7tky7IsyU7ii+wkdkgg5KbjJIQJmWSTmBwWcxtwgEwYOJPlkrBcd8MOGzJmZsnA7DC7kAEC4yHAEMNwCYb1rMmSGzMkxDJJnNghtuL4otio25JltSWrdXv2jy7JnbZktaLurpL69zlHx91Vb6mfLkm/fv3WW1Xm7oiISHEoCbsAEREpHIW+iEgRUeiLiBQRhb6ISBFR6IuIFBGFvohIEVHoi4gUEYW+iEgRUeiLiBSRsrALyNTY2OjLly8PuwwRkRll27Zth929abJ2kQv95cuX09raGnYZIiIzipnty6adhndERIqIQl9EpIgo9EVEiohCX0SkiCj0RUSKiEJfRKSIKPRFRIpI5Obpi4gUm4Pdx3l0V5xhd95zaXNeXyur0DezNcD/AkqBb7n7XRnrm4ENQBPQBbzX3duDdcPAM0HT/e7+5hzVLiIyI/UPDvP4nk4e3XWYR3fHaYsdA+CiZfPDD30zKwXuBq4F2oGtZrbJ3XemNftb4Dvufq+ZXQ18AbgpWHfc3S/Mcd0iIjOGu7M7doxHd8V5ZFecJ17sIjk0QkVZCZeuWMC7WpZy5aomVi2am/dasunprwba3H0PgJltBNYC6aF/HvDx4PFDwP25LFJEJN8S/YPs6+zjpe7jVJeXUlddzvyacuZXV1BbVUZJiU3p+3X3DfBvbYd5dFecX+8+zKGj/QCcvXAu77m0mStXNXLpigaqK0rz8XYmlE3oLwYOpD1vBy7NaPM08HZSQ0BvBWrNrMHdO4EqM2sFhoC73F0fCCJySge6+jhwpI95VUHw1lQwp6IUs6kFbzp3p6t3gH1dfezr7GVfZx/7OvvY29nL/s4+OnsHJtzWjBO1VJdTV1OR+lAIPhjqqlNftVXlPHeoh0d3x3n6QDcjDrVVZbz+7EY+ek0TV65qYvH86lf8HnIhm9Afby97xvNPAV81s/cBjwIvkQp5gGXuftDMzgQeNLNn3P2Fl72A2S3ALQDLli2bQvkiMhv0DQyNjXE/sivOi4d7T2pTVmIvC9j5NRVBAKd642PrasqpKC2h/UjfWLDv6+pl3+E+Esmhse9nBmfUVdPcUMN1r15Ec8McmhfUsKS+hoHhYbr7BlNfxwc5enyQo30DdB9PLTt6fJADXX109w1w9PggI2mJaAYXLJnPrVev5A2rGrlgyXzKSqMzUTKb0G8HlqY9XwIcTG/g7geBtwGY2Vzg7e5+NG0d7r7HzB4GLgJeyNj+HuAegJaWlswPFBGZZdyd5w4leHR3nEd3xWnde4SB4RGqyku47MwGbrqsmVedVktP/xBHjw+MBW338UGO9g3SfXyAWKKfXR0JjvYNvizM05WVGEsX1LBsQQ2XLKtPBXtDDc0Nc1hSX01V+fSHVkZGnGMDQxwNalw8v5r6ORXT/r75kk3obwVWmtkKUj34dcC70xuYWSPQ5e4jwGdIzeTBzOqBPndPBm2uAL6Yw/pFJE/cnSN9g5SVGrWVZdMaWgHoPJbk39pSPflf7z5MPJEE4FWn1fK+K5Zz5comWpbXv6IgHhoeoad/iO6gN94/OMzS+hpOr6vKey+7pMSYV1XOvKryl/WOo2rS0Hf3ITO7FdhCasrmBnffYWbrgVZ33wRcBXzBzJzU8M5Hgs3PBb5hZiOkTgS7K2PWj4hkITk0TEVpybSDN9PIiPOHnv6xce29nenj3b30DgwDUFpiY2PY86pPjG3Pr6lIPR9dNjb8khrzfvFw79iMlWcPHsUd5teU8/qzG7lyVRNXrmzitLqqab+PstISFsypYEGEe9hRYe7RGk1paWlx3URFJDWX+xfbD/Hdx/bydPtRKspKMg4cVpw4sDi6fHScO23mSVVFCYe6g2Dv6mPv4T72d/Wyt7OP/V19DAyNjL1meamxtL6GZQ01LA+GQNyh+3hq7HpsmCUYYjnaN0hP//hDK6NKS4yLls5PhfyqJs5fXEfpFGfCyOTMbJu7t0zWTmfkikTMga4+vvf4Pn7QeoDuvkHOXjiXj159NsnhkbFx4+6+QV7qPs7Og0c5enxwrEeejeryUpobajiraQ5Xv2phaox7QWqs+4z51VMO5OERp+f4iQOeowc3u/sGWTSvksvPaqSuunyqu0HyRKEvEgEjI84ju+J857G9PLwrTokZ1523iJsub+byMxsmHdYZGBpJzTBJ65GPzjzpSw5xWl0VzQ1zWN5QQ1NtZU6HiUpLjPo5FZE+eCknKPRFQnSkd4B/2XaA7z2+n/1dfTTOreS2Pz6bGy9dxul12c/nrigroam2kqbayjxWK7OBQl8Kyt355q/38JPfvRR2Ka/IgjkVY1P+mhfUjE0BnFM5tT+l7e3dfOexffz86YMkh0ZYvWIBn77+HK5/9WlUlEVnTrfMPgp9KZj+wWFu//F27n/qIJc019M4d2YNB7hD/FiSX+7oOOnszca5lSxvOHEAdPSDYXlDDfNrUu9z7MDs4/t4+kA3NRWlvOOSJdx0eTOvOm1eGG9JipBCXwoilujnP313G0/u7+aT167i1qvPzvn0w0IavU5L+mn8ezt7eeyFzpP+FzOvqozmhjm0H+njSN8gZzXN4S/f/GreevFi5lXpAKcUlkJf8m7HwaP8+b2tdPUN8LX3XMwbzz897JKmrbaqnNcsruM1i+tOWtc/OMyBrpfPed/b2UtzQyPvXr2My8+a/MCsSL4o9CWvtuz4Ax/b+BR11eX86IOvGzckZ5uq8lJWLqpl5aLasEsROYlCX/LC3fmHh1/gS1ue54IldXzzT1tYOG/6Z16KyPQo9CXn+geH+cxPnuGnT77Ef7zgDL70jtfm5MJWIjJ9Cn3Jqdl2wFZktlHoS87MxgO2IrONQl9yohgP2IrMRAr9Ira/s4+B4RGWLqimsuyVjbnrgK3IzKLQL0Luzj/9+17+6v/sZMRfftu4qVxiQAdsRWYehX6RGRga4XObnuW+Jw5w7XmLuOH804Lrq6dOIJroEgNjHwgL5rC8sYbT5lVx1//9PU/u7+YT167iNh2wFZkRFPpF5EjvAB/83jZ++2IXH77qLD513TmUjHPt9PRLDIzeUHpf18mXGKgqL9EBW5EZRqFfJNpiCT5wbyuHuvv5u3dewNsuXjJh22wuMbCvs4+zF85leeOcfJYtIjmm0C8CDz8f47bvP0lleQn33XIZlzTXv+LvpUsMiMxsCv1ZLP2A7apFtXzr5haW1NeEXZaIhEihP0sNDo9wx892cN8T+7nuvEV8+V0XTvlGHyIy+2R1ix4zW2Nmz5tZm5ndPs76ZjP7lZltN7OHzWxJ2rqbzWx38HVzLouX8R3pHeCmf/wt9z2xnw9fdRZff+8lCnwRAbLo6ZtZKXA3cC3QDmw1s03uvjOt2d8C33H3e83sauALwE1mtgD4HNACOLAt2PZIrt+IpEzlgK2IFJ9sevqrgTZ33+PuA8BGYG1Gm/OAXwWPH0pbfz3wgLt3BUH/ALBm+mXLeB7ZFeetd/+G3uQQ991ymQJfRE6STegvBg6kPW8PlqV7Gnh78PitQK2ZNWS5rUxT6oDti/zZPz3B4vpq7v/IFdOaoSMis1c2oT/eaZae8fxTwBvM7EngDcBLwFCW22Jmt5hZq5m1xuPxLEqSUYPDI/y3nz7LX/58J9ecu4gff+h1mqEjIhPK5uheO7A07fkS4GB6A3c/CLwNwMzmAm9396Nm1g5clbHtw5kv4O73APcAtLS0nPShIONrix3js/c/w+N7uvjQVWfx6QnOsBURGZVN6G8FVprZClI9+HXAu9MbmFkj0OXuI8BngA3Bqi3A/zCz0bGG64L1Mg3PHerhqw+1sfmZQ1SVleqArYhkbdLQd/chM7uVVICXAhvcfYeZrQda3X0Tqd78F8zMgUeBjwTbdpnZ50l9cACsd/euPLyPorC9vZuvPNjGAzs7mFtZxofecBYfeP0KGuZWhl2aiMwQ5h6t0ZSWlhZvbW0Nu4xIad3bxVcebOORXXHqqst5/xUreN/rllNXUx52aSISEWa2zd1bJmunM3Yiyt157IVOvvJgG4/t6WTBnAr+y5pzuOmyZmqrFPYi8soo9CPG3Xl4V5yvPtjGtn1HWFhbyX9/03ncuHopNRX6cYnI9ChFImJkxHnguQ6++mAbz7x0lMXzq/n8W17Dn1yyRHejEpGcUeiHzN35xfZD3P1QG7//Q4Lmhhq++PbX8paLFlNRltWlkUREsqbQD9nPtx/io/c9ydkL5/L377qQN732dMpKFfYikh8K/ZD9/lAPZSXGlo9dSalOrBKRPFOXMmSxRJKm2koFvogUhEI/ZLFEkoW1OrlKRApDoR+yWE8/C+dVhV2GiBQJhX7I1NMXkUJS6IdoYGiErt4BFtaqpy8ihaHQD9HhY0kAFs5TT19ECkOhH6KOnn4AFin0RaRAFPohiiWCnr6Gd0SkQBT6IToR+urpi0hhKPRDFO/pp8TQTVBEpGAU+iHq6EnSOFdn44pI4Sj0QxRL9GvmjogUlEI/RKkTs3QQV0QKR6EfIp2NKyKFptAPydDwCJ3HFPoiUlhZhb6ZrTGz582szcxuH2f9MjN7yMyeNLPtZnZDsHy5mR03s6eCr6/n+g3MVJ29A4w4utiaiBTUpDdRMbNS4G7gWqAd2Gpmm9x9Z1qzzwI/dPevmdl5wGZgebDuBXe/MLdlz3yxHs3RF5HCy6anvxpoc/c97j4AbATWZrRxYF7wuA44mLsSZ6dYInUJBvX0RaSQsgn9xcCBtOftwbJ0dwLvNbN2Ur3829LWrQiGfR4xsz+aTrGzic7GFZEwZBP645055BnPbwS+7e5LgBuA75pZCXAIWObuFwGfAL5vZvMytsXMbjGzVjNrjcfjU3sHM9ToxdaaFPoiUkDZhH47sDTt+RJOHr75APBDAHd/DKgCGt096e6dwfJtwAvAqswXcPd73L3F3Vuampqm/i5moFgiScOcCspLNYFKRAonm8TZCqw0sxVmVgGsAzZltNkPXANgZueSCv24mTUFB4IxszOBlcCeXBU/k8V6kurli0jBTTp7x92HzOxWYAtQCmxw9x1mth5odfdNwCeBb5rZx0kN/bzP3d3MrgTWm9kQMAx80N278vZuZpB4QvfGFZHCmzT0Adx9M6kDtOnL7kh7vBO4Ypztfgz8eJo1zkodPUlWLaoNuwwRKTIaUA7ByIhz+FhSF1sTkYJT6Iegq2+AoRHXxdZEpOAU+iHQ2bgiEhaFfgg6dDauiIREoR+CuHr6IhIShX4IRq+7o3n6IlJoCv0QxBJJ6qrLqSovDbsUESkyCv0QdPT0s0jTNUUkBAr9EOjeuCISFoV+CGI9uk2iiIRDoV9g7k48kaRJwzsiEgKFfoF19w0yMDzCIg3viEgIFPoFNnbHLPX0RSQECv0CG7s3rnr6IhIChX6B6bo7IhImhX6BnbjujkJfRApPoV9gsZ4ktZVl1FRkdf8aEZGcUugXmKZrikiYFPoFFkv0azxfREKj0C+wjh5dgkFEwqPQLyB3J5bQxdZEJDxZhb6ZrTGz582szcxuH2f9MjN7yMyeNLPtZnZD2rrPBNs9b2bX57L4mSaRHKJ/cEQ9fREJzaRTSMysFLgbuBZoB7aa2SZ335nW7LPAD939a2Z2HrAZWB48Xge8GjgD+H9mtsrdh3P9RmaCsTn66umLSEiy6emvBtrcfY+7DwAbgbUZbRyYFzyuAw4Gj9cCG9096e4vAm3B9ytKsR7dMUtEwpVN6C8GDqQ9bw+WpbsTeK+ZtZPq5d82hW2Lxuh1dxbphugiEpJsQt/GWeYZz28Evu3uS4AbgO+aWUmW22Jmt5hZq5m1xuPxLEqamU5cd0c9fREJRzah3w4sTXu+hBPDN6M+APwQwN0fA6qAxiy3xd3vcfcWd29pamrKvvoZJtaTpLq8lLmVOhtXRMKRTehvBVaa2QozqyB1YHZTRpv9wDUAZnYuqdCPB+3WmVmlma0AVgJP5Kr4maYjkWThvErMxvsPkIhI/k3a5XT3ITO7FdgClAIb3H2Hma0HWt19E/BJ4Jtm9nFSwzfvc3cHdpjZD4GdwBDwkWKduQOpA7m6eYqIhCmrcQZ330zqAG36sjvSHu8Erphg278G/noaNc4a8USSc8+YN3lDEZE80Rm5BRRL6IboIhIuhX6B9CaHOJYc0tm4IhIqhX6BnJijr56+iIRHoV8go2fjqqcvImFS6BfIaE9f190RkTAp9Auko0dn44pI+BT6BRJPJKkoK6GuujzsUkSkiCn0C2R0uqbOxhWRMCn0C0T3xhWRKFDoF0hM98YVkQhQ6BdIR4/ujSsi4VPoF0D/4DA9/UMs1M1TRCRkCv0CiAdz9HWbRBEJm0K/AHTHLBGJCoV+AXT06N64IhINCv0CiOlsXBGJCIV+AcQSScpKjPqairBLEZEip9AvgFgiSVNtJSUlOhtXRMKl0C+Ajh6djSsi0aDQL4B4Iqk5+iISCQr9AtC9cUUkKhT6eTYwNEJX74CuuyMikZBV6JvZGjN73szazOz2cdZ/2cyeCr52mVl32rrhtHWbcln8TBA/pjtmiUh0lE3WwMxKgbuBa4F2YKuZbXL3naNt3P3jae1vAy5K+xbH3f3C3JU8s4zO0dfF1kQkCrLp6a8G2tx9j7sPABuBtadofyNwXy6Kmw3G7o2r4R0RiYBsQn8xcCDteXuw7CRm1gysAB5MW1xlZq1m9riZvWWC7W4J2rTG4/EsS58ZToS+evoiEr5sQn+8M4p8grbrgB+5+3DasmXu3gK8G/h7MzvrpG/mfo+7t7h7S1NTUxYlzRyxnn5KDBrmKvRFJHzZhH47sDTt+RLg4ARt15ExtOPuB4N/9wAP8/Lx/lkv1pOkcW4lpTobV0QiIJvQ3wqsNLMVZlZBKthPmoVjZucA9cBjacvqzawyeNwIXAHszNx2Nosl+jVzR0QiY9LZO+4+ZGa3AluAUmCDu+8ws/VAq7uPfgDcCGx09/Shn3OBb5jZCKkPmLvSZ/0Ug1giqUsqi0hkTBr6AO6+GdicseyOjOd3jrPdb4Dzp1HfjNfRk+T8xXVhlyEiAuiM3LwaGh6hs1fX3RGR6FDo51Fn7wDumq4pItGh0M+jWI/m6ItItCj086hj9DaJGt4RkYhQ6OfR6Nm4uu6OiESFQj+PYol+zKBRZ+OKSEQo9PMolkiyoKaC8lLtZhGJBqVRHsV6+mnSQVwRiRCFfh7pbFwRiRqFfh7FenRvXBGJFoV+noyMOPFjSV1sTUQiRaGfJ529AwyPuO6YJSKRotDPk1hC98YVkehR6OfJ6IlZTerpi0iEKPTzJK7r7ohIBCn082T0ujuapy8iUaLQz5NYIklddTlV5aVhlyIiMkahnyexRL8O4opI5Cj08ySWSGq6pohEjkI/T3Q2rohEkUI/D9ydeCJJk4Z3RCRisgp9M1tjZs+bWZuZ3T7O+i+b2VPB1y4z605bd7OZ7Q6+bs5l8VHV3TfIwPAIizS8IyIRUzZZAzMrBe4GrgXaga1mtsndd462cfePp7W/DbgoeLwA+BzQAjiwLdj2SE7fRcSMnpil6+6ISNRk09NfDbS5+x53HwA2AmtP0f5G4L7g8fXAA+7eFQT9A8Ca6RQ8E4xegkEHckUkarIJ/cXAgbTn7cGyk5hZM7ACeHAq25rZLWbWamat8Xg8m7ojrUNn44pIRGUT+jbOMp+g7TrgR+4+PJVt3f0ed29x95ampqYsSoq2sZ6+hndEJGKyCf12YGna8yXAwQnaruPE0M5Ut501Yj1JaivLqKmY9JCJiEhBZRP6W4GVZrbCzCpIBfumzEZmdg5QDzyWtngLcJ2Z1ZtZPXBdsGxW03RNEYmqSbui7j5kZreSCutSYIO77zCz9UCru49+ANwIbHR3T9u2y8w+T+qDA2C9u3fl9i1ET0dPv8bzRSSSshp/cPfNwOaMZXdkPL9zgm03ABteYX0zUiyR5KJl88MuQ0TkJDojN8fcnVhCPX0RiSaFfo4lkkP0D45ojr6IRJJCP8diPZquKSLRpdDPsdjYiVnq6YtI9Cj0c0zX3RGRKFPo59iJ6+4o9EUkehT6OdbRk6S6vJS5lTobV0SiR6GfY7FEkkXzKjEb77JDIiLhUujnWKynXwdxRSSyFPo5puvuiEiUKfRzTNfdEZEoU+jnUG9yiN6BYRbN0/COiESTQj+Hxuboq6cvIhGl0M+hsUsw6ECuiESUQj+HOnQ2rohEnEI/h0709BX6IhJNCv0ciieSVJSVUFddHnYpIiLjUujnUCyRZGGtzsYVkehS6OeQ5uiLSNQp9HMo1dPXzB0RiS6Ffg7FevpZpJk7IhJhWYW+ma0xs+fNrM3Mbp+gzTvNbKeZ7TCz76ctHzazp4KvTbkqPGr6B4fp6R9ioc7GFZEIm/Si72ZWCtwNXAu0A1vNbJO770xrsxL4DHCFux8xs4Vp3+K4u1+Y47ojZ/Q2iU0a0xeRCMump78aaHP3Pe4+AGwE1ma0+XPgbnc/AuDusdyWGX26Y5aIzATZhP5i4EDa8/ZgWbpVwCoz+3cze9zM1qStqzKz1mD5W8Z7ATO7JWjTGo/Hp/QGomL0uju62JqIRFk29/Qbb9K5j/N9VgJXAUuAX5vZa9y9G1jm7gfN7EzgQTN7xt1feNk3c78HuAegpaUl83vPCDobV0Rmgmx6+u3A0rTnS4CD47T5mbsPuvuLwPOkPgRw94PBv3uAh4GLpllzJHUkkpSVGPU1FWGXIiIyoWxCfyuw0sxWmFkFsA7InIVzP/DHAGbWSGq4Z4+Z1ZtZZdryK4CdzEKxniRNtZWUlOhsXBGJrkmHd9x9yMxuBbYApcAGd99hZuuBVnffFKy7zsx2AsPAp92908xeB3zDzEZIfcDclT7rZzaJJfo1XVNEIi+bMX3cfTOwOWPZHWmPHfhE8JXe5jfA+dMvM/riiSRLF9SEXYaIyCnpjNwc0XV3RGQmUOjnwMDQCEf6BnXdHRGJPIV+DsSPjc7RV09fRKJNoZ8DY3P0FfoiEnEK/RzoCK67o+EdEYk6hX4OxHXdHRGZIRT6ORBLJCkxaJir0BeRaMtqnv5M0N03wJ98/bFQXjuWSNI4t5JSnY0rIhE3a0K/pMRYuWhuKK+9ctFcLj+zIZTXFhGZilkT+vOqyvmH91wSdhkiIpGmMX0RkSKi0BcRKSIKfRGRIqLQFxEpIgp9EZEiotAXESkiCn0RkSKi0BcRKSKWutNhdJhZHNg3jW/RCBzOUTm5pLqmRnVNjeqamtlYV7O7N03WKHKhP11m1uruLWHXkUl1TY3qmhrVNTXFXJeGd0REiohCX0SkiMzG0L8n7AImoLqmRnVNjeqamqKta9aN6YuIyMRmY09fREQmMCND38zWmNnzZtZmZrePs77SzH4QrP+tmS0vQE1LzewhM3vOzHaY2X8ep81VZnbUzJ4Kvu7Id11pr73XzJ4JXrd1nPVmZv872GfbzeziAtR0Ttq+eMrMeszsYxltCrLPzGyDmcXM7Nm0ZQvM7AEz2x38Wz/BtjcHbXab2c0FqOtLZvb74Of0UzObP8G2p/yZ56GuO83spbSf1Q0TbHvKv9881PWDtJr2mtlTE2ybz/01bj6E8jvm7jPqCygFXgDOBCqAp4HzMtp8GPh68Hgd8IMC1HU6cHHwuBbYNU5dVwG/CGm/7QUaT7H+BuBfAQMuA34bws/1D6TmGhd8nwFXAhcDz6Yt+yJwe/D4duBvxtluAbAn+Lc+eFyf57quA8qCx38zXl3Z/MzzUNedwKey+Dmf8u8313VlrP+fwB0h7K9x8yGM37GZ2NNfDbS5+x53HwA2Amsz2qwF7g0e/wi4xszyegNbdz/k7r8LHieA54DF+XzNHFsLfMdTHgfmm9npBXz9a4AX3H06J+a9Yu7+KNCVsTj99+he4C3jbHo98IC7d7n7EeABYE0+63L3X7r7UPD0cWBJrl5vOnVlKZu/37zUFWTAO4H7cvV62TpFPhT8d2wmhv5i4EDa83ZODtexNsEfx1GgYDexDYaTLgJ+O87qy83saTP7VzN7daFqAhz4pZltM7NbxlmfzX7Np3VM/McY1j5b5O6HIPVHCywcp03Y++39pP6HNp7Jfub5cGsw7LRhgqGKMPfXHwEd7r57gvUF2V8Z+VDw37GZGPrj9dgzpyBl0yYvzGwu8GPgY+7ek7H6d6SGLy4AvgLcX4iaAle4+8XAG4GPmNmVGevD3GcVwJuBfxlndZj7LBth7re/AIaAf56gyWQ/81z7GnAWcCFwiNRQSqbQ9hdwI6fu5ed9f02SDxNuNs6yV7zPZmLotwNL054vAQ5O1MbMyoA6Xtl/RafEzMpJ/UD/2d1/krne3Xvc/VjweDNQbmaN+a4reL2Dwb8x4Kek/pudLpv9mi9vBH7n7h2ZK8LcZ0DH6BBX8G9snDah7LfgYN6bgPd4MPCbKYufeU65e4e7D7v7CPDNCV4vrP1VBrwN+MFEbfK9vybIh4L/js3E0N8KrDSzFUEPcR2wKaPNJmD0CPc7gAcn+sPIlWC88B+B59z97yZoc9rosQUzW01q/3fms67gteaYWe3oY1IHAp/NaLYJ+FNLuQw4OvrfzgKYsAcW1j4LpP8e3Qz8bJw2W4DrzKw+GM64LliWN2a2BvivwJvdvW+CNtn8zHNdV/oxoLdO8HrZ/P3mw38Afu/u7eOtzPf+OkU+FP53LB9HqvP9RWqmyS5SswD+Ili2ntQfAUAVqaGCNuAJ4MwC1PR6Uv/l2g48FXzdAHwQ+GDQ5lZgB6kZC48DryvQ/jozeM2ng9cf3WfptRlwd7BPnwFaClRbDakQr0tbVvB9RupD5xAwSKpn9QFSx4F+BewO/l0QtG0BvpW27fuD37U24M8KUFcbqTHe0d+z0ZlqZwCbT/Uzz3Nd3w1+d7aTCrPTM+sKnp/095vPuoLl3x5i2U/wAAAASElEQVT9nUprW8j9NVE+FPx3TGfkiogUkZk4vCMiIq+QQl9EpIgo9EVEiohCX0SkiCj0RUSKiEJfRKSIKPRFRIqIQl9EpIj8f4oDarr3gvfiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.62741652, 0.62741652, 0.62741652, 0.88400703, 0.89630931,\n",
       "       0.91739895, 0.91388401, 0.93497364, 0.93497364, 0.93497364,\n",
       "       0.93497364, 0.9314587 , 0.94200351, 0.94024605, 0.94376098,\n",
       "       0.94200351, 0.94903339, 0.94727592, 0.95079086, 0.94903339,\n",
       "       0.95606327])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Reference"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[sklearn.tree.DecisionTreeRegressor -- scikit-learn 0.21.3 documentation](https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeRegressor.html)\n",
    "\n",
    "[Understanding the decision tree structure -- scikit-learn 0.21.3 documentation](https://scikit-learn.org/stable/auto_examples/tree/plot_unveil_tree_structure.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
